import scrapy

import logging
from copy import deepcopy, copy
import json
from HifoEsf.items import CommunityItem, CaseItem
import re

logger = logging.getLogger(__name__)


class TmsfSpider(scrapy.Spider):
    name = 'tmsf'
    allowed_domains = ['tmsf.com']
    start_urls = ['https://www.tmsf.com/esf/esfnSearch_communityListUpgrade.htm']
    hz_project_list_url = 'https://www.tmsf.com/esfn/EsfnSearch_queryCommunityList.jspx'  # POST 【杭州】小区列表
    hz_project_detail_url = 'https://www.tmsf.com/esf/xq_index_{}.htm'  # GET 【杭州】小区详情
    hz_case_list_url = 'https://www.tmsf.com/esfn/EsfnSearch_sellhousNewList.jspx'  # POST 【杭州】案例列表
    hz_case_list_referer = 'https://www.tmsf.com/esfn/EsfnSearch_csnewInfo.jspx?communityid={}'  # 【杭州】案例列表
    hz_case_detail_url = 'https://www.tmsf.com/esf/xq_csUpgrade_{}.htm'  # 【杭州】案例详情

    custom_settings = {
        'DOWNLOADER_MIDDLEWARES': {
            'scrapy.downloadermiddlewares.retry.RetryMiddleware': None,
            'HifoEsf.middlewares.CustomRetryMiddleware': 500,
            'HifoEsf.middlewares.UserAgentMiddleware': 544,
            'HifoEsf.middlewares.OuterNetProxyMiddleware': 545,  # 启用redis中的代理，需要同时开启 RedisConnPipeline
            'HifoEsf.middlewares.TmsfCookiesMiddleware': 546,
        },
        'ITEM_PIPELINES': {
            'HifoEsf.pipelines.RedisConnPipeline': 299,  # 启用redis中的代理，需要同时开启 XXXProxyMiddleware
            'HifoEsf.pipelines.TmsfPipeline': 301,
            'HifoEsf.pipelines.MongoClientPipeline': 399,
        },
        'RETRY_HTTP_CODES': [500, 502, 503, 504, 400, 404, 408, 407, 302, 403],
        'RETRY_TIMES': 20,
        'COOKIES_ENABLED': True,  # 携带cookies
        'COOKIES_REDIS_NAME': 'tmsf_cookies',
    }

    def start_requests(self):
        # 杭州市
        # 构造POST请求，获取小区列表首页
        page_num = 1
        form_data: dict = {
            'aid': '',
            'pr': '',
            'ctype': '',
            'age': '',
            'ordertype': '',
            'keywords': '',
            'page': str(page_num),
            'rp': str(10),
        }
        headers: dict = {
            'Referer': self.start_urls[0],
        }
        yield scrapy.FormRequest(
            self.hz_project_list_url,
            headers=headers,
            formdata=form_data,
            meta=dict(page_num=deepcopy(page_num), ),
            dont_filter=True,
            priority=10,
        )

    def parse(self, response, **kwargs):
        """
        杭州市
        获取小区列表页
        :param response:
        :param kwargs:
        :return:
        """
        page_num = copy(response.meta['page_num'])
        # 提取小区列表
        try:
            resp_dict = json.loads(response.body.decode())
            project_list = resp_dict.get('list', list())
            total_page_num = resp_dict.get('page', dict()).get('total')
        except Exception as e:
            logging.error('杭州市-第{}页 小区列表提取出错,error:{}'.format(page_num, e))
        else:
            # 遍历提取小区信息
            for project_dict in project_list:
                item_cmty = CommunityItem()
                item_cmty['provinceName'] = '浙江'  # 省份
                item_cmty['cityName'] = '杭州'  # 城市
                item_cmty['districtName'] = self.get_district_name(project_dict.get("arearmk"))  # 行政区
                item_cmty['shangQuan'] = project_dict.get("areaname")  # 商圈
                item_cmty['name'] = project_dict.get("communityabb")  # 楼盘名称
                item_cmty['otherName'] = project_dict.get("communityname")  # 其他名称
                item_cmty['guid'] = project_dict.get("communityid")  # 楼盘id
                item_cmty['address'] = project_dict.get("communityaddr")  # 楼盘地址
                item_cmty['buildYear'] = project_dict.get("doneyear")  # 建筑年代
                item_cmty['price'] = project_dict.get("gpprice")  # 单价
                sign_price = project_dict.get("signprice")
                item_cmty['other'] = f'签约单价:{sign_price}元' if sign_price else None  # 签约单价
                # 构造请求，获取小区详情
                if item_cmty['guid']:
                    yield scrapy.Request(
                        self.hz_project_detail_url.format(item_cmty['guid']),
                        meta=dict(item_cmty=deepcopy(item_cmty), ),
                        callback=self.parse_project_detail,
                        priority=20,
                    )
                else:
                    logger.error('杭州-第{}页-{} 小区id提取为空'.format(page_num, item_cmty['name']))

            # 翻页
            if page_num == 1 and total_page_num > 1:
                for next_page_num in range(2, total_page_num + 1):
                    # 构造POST请求，获取小区列表页
                    form_data: dict = {
                        'aid': '',
                        'pr': '',
                        'ctype': '',
                        'age': '',
                        'ordertype': '',
                        'keywords': '',
                        'page': str(next_page_num),
                        'rp': str(10),
                    }
                    headers: dict = {
                        'Referer': self.start_urls[0],
                    }
                    yield scrapy.FormRequest(
                        self.hz_project_list_url,
                        headers=headers,
                        formdata=form_data,
                        meta=dict(page_num=deepcopy(next_page_num), ),
                        dont_filter=True,
                        priority=10,
                    )

    def parse_project_detail(self, response):
        """
        杭州市
        获取小区详情
        :param response:
        :return:
        """
        item_cmty = copy(response.meta['item_cmty'])
        item_cmty['propertyFee'] = response.xpath(
            "//span[contains(text(),'物业费用')]/following-sibling::span[1]/text()").extract_first()  # 物管费
        item_cmty['grossBuildArea'] = response.xpath(
            "//span[contains(text(),'建筑面积')]/following-sibling::span[1]/text()").extract_first()  # 楼盘建筑面积
        item_cmty['type'] = response.xpath(
            "//span[contains(text(),'物业类型')]/following-sibling::span[1]/text()").extract_first()  # 物业类型
        item_cmty['numberOfParkingSpaces'] = response.xpath(
            "//span[contains(text(),'车 位 数')]/following-sibling::span[1]/text()").extract_first()  # 车位数
        item_cmty['totalRoomCount'] = response.xpath(
            "//span[contains(text(),'总 户 数')]/following-sibling::span[1]/text()").extract_first()  # 总户数
        item_cmty['floorAreaRatio'] = response.xpath(
            "//span[contains(text(),'容 积 率')]/following-sibling::span[1]/text()").extract_first()  # 容积率
        item_cmty['greeningRate'] = response.xpath(
            "//span[contains(text(),'绿 化 率')]/following-sibling::span[1]/text()").extract_first()  # 绿化率
        item_cmty['propertyCompany'] = response.xpath(
            "//span[contains(text(),'物业公司')]/following-sibling::div[1]/div/text()").extract_first()  # 物管公司
        item_cmty['monthOnMonth'] = response.xpath(
            "//a[contains(text(),'查看市场行情')]/preceding-sibling::font/text()").extract_first()  # 环比
        item_cmty['location'] = self.get_location(
            response.xpath("//input[@id='pageLoad']/@value").extract_first())  # 经纬度
        yield item_cmty

        # 构造请求，获取案例列表首页
        page_num = 1
        form_data: dict = {
            'wawjrp': str(6),
            'rp': str(16),
            'communityid': str(item_cmty['guid']),
        }
        headers: dict = {
            'Referer': self.hz_case_list_referer.format(item_cmty['guid']),
        }
        yield scrapy.FormRequest(
            self.hz_case_list_url,
            headers=headers,
            formdata=form_data,
            callback=self.parse_case_list,
            meta=dict(item_cmty=deepcopy(item_cmty), page_num=deepcopy(page_num), ),
            dont_filter=True,
            priority=30,
        )

    def parse_case_list(self, response):
        """
        杭州
        获取案例列表
        :param response:
        :return:
        """
        item_cmty = copy(response.meta['item_cmty'])
        page_num = copy(response.meta['page_num'])
        try:
            resp_dict = json.loads(response.body.decode())
            case_list = resp_dict.get('list', list())
            total_page_num = int(resp_dict.get('total'))
        except Exception as e:
            logging.error(
                '杭州市-{}-{}-第{}页 案例列表提取出错,error:{}'.format(item_cmty['districtName'], item_cmty['name'], page_num, e))
        else:
            # 遍历提取案例信息
            for case_dict in case_list:
                item_case = CaseItem()
                item_case['provinceName'] = item_cmty['provinceName']  # 省份
                item_case['cityName'] = item_cmty['cityName']  # 城市
                item_case['districtName'] = item_cmty['districtName']  # 行政区
                item_case['shangQuan'] = item_cmty['shangQuan']  # 商圈
                item_case['name'] = item_cmty['name']  # 楼盘名称
                item_case['guid'] = item_cmty['guid']  # 小区id
                item_case['caseType'] = '挂牌'  # 案例类型
                item_case['title'] = case_dict.get('fymc')  # 标题
                item_case['caseId'] = case_dict.get('fwcode')  # 案例id
                item_case['dataUrl'] = self.hz_case_detail_url.format(item_case['caseId']) if item_case[
                    'caseId'] else None  # 案例url
                item_case['floor'] = case_dict.get('floortypeValue')  # 所在楼层区
                item_case['floorLevels'] = case_dict.get('szlc')  # 所在楼层数
                item_case['totalFloor'] = case_dict.get('zzcs')  # 总楼层
                item_case['directionType'] = case_dict.get('fwcxValue')  # 朝向
                item_case['calQuality'] = case_dict.get('fwytvalue')  # 住宅类别
                item_case['buildArea'] = case_dict.get('jzmj')  # 建筑面积
                item_case['buildYear'] = case_dict.get('jznd')  # 建筑年代
                item_case['totalPrice'] = case_dict.get('price')  # 总价
                item_case['price'] = case_dict.get('singleprice')  # 单价
                item_case['otherAddress'] = case_dict.get('communityaddr')  # 其他地址
                item_case['decorationType'] = case_dict.get('zxbzValue')  # 装修
                item_case['educationFacility'] = case_dict.get('school')  # 教育配套
                item_case['transportFacility'] = case_dict.get('subwaymark')  # 交通状况
                item_case['updateDate'] = case_dict.get('updatetime')  # 更新时间
                item_case['listingDate'] = case_dict.get('hysj')  # 挂牌日期
                item_case['createDate'] = case_dict.get('scgpshsj')  # 创建时间
                item_case['bedroom'] = case_dict.get('hxs')  # 室
                item_case['livingroom'] = case_dict.get('hxt')  # 厅
                item_case['bathroom'] = case_dict.get('hxw')  # 卫
                price_spread = case_dict.get('pricespread')
                item_case['other'] = '价格变化:{}万'.format(price_spread) if price_spread else None  # 其它信息
                yield item_case

            # 翻页
            if page_num == 1 and total_page_num > 1:
                for next_page_num in range(2, total_page_num + 1):
                    form_data: dict = {
                        'wawjrp': str(6),
                        'rp': str(16),
                        'page': str(next_page_num),
                        'communityid': str(item_cmty['guid']),
                    }
                    headers: dict = {
                        'Referer': self.hz_case_list_referer.format(item_cmty['guid']),
                    }
                    yield scrapy.FormRequest(
                        self.hz_case_list_url,
                        headers=headers,
                        formdata=form_data,
                        callback=self.parse_case_list,
                        meta=dict(item_cmty=deepcopy(item_cmty), page_num=deepcopy(next_page_num), ),
                        dont_filter=True,
                        priority=30,
                    )

    @staticmethod
    def get_district_name(area):
        """
        提取行政区
        :param area:
        :return:
        """
        try:
            assert area
            district_name = area.split(' ')[0].strip()
            assert district_name
        except:
            return
        else:
            return district_name

    @staticmethod
    def get_location(value: str):
        """
        提取小区经纬度
        :param value:
        :return:
        """
        try:
            assert value
            lng = float(re.findall(r'\d{3}\.\d+', value)[0])
            lat = float(re.findall(r'\d{2}\.\d+', value)[0])
            assert lng and lat
        except:
            return None, None
        else:
            return lng, lat
